import warnings
warnings.filterwarnings('ignore')
pd.set_option('float_format', '{:f}'.format)
pd.set_option('display.max_columns', None)

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
from scipy.stats import norm
import pandas_profiling
import numpy as np
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans
np.random.seed(sum(map(ord, "aesthetics")))
from sklearn import metrics

import seaborn as sns

#loading csv
iris_dataset = pd.read_csv('/Users/rajmati.marlecha/Desktop/DMGAssignment/iris-species/Iris.csv')
iris_dataset.head()

Question 1 :¶

#1 Histograms without labels for all 4 dimensions
f, (ax1,ax2,ax3,ax4) = plt.subplots(1, 4, figsize=(20,4))
    
ax1.set_title('SepalLengthCm')
ax2.set_title('SepalWidthCm')
ax3.set_title('PetalLengthCm')
ax4.set_title('PetalWidthCm')
    
sns.distplot(iris_dataset['SepalLengthCm'],ax=ax1)   
sns.distplot(iris_dataset['SepalWidthCm'],ax=ax2) 
sns.distplot(iris_dataset['PetalLengthCm'],ax=ax3) 
sns.distplot(iris_dataset['PetalWidthCm'],ax=ax4) 
    
plt.show()

#1 Histograms with labels : Species for all 4 dimensions
g = sns.FacetGrid(iris_dataset, col="Species")
g = g.map(plt.hist, "SepalLengthCm")
g = sns.FacetGrid(iris_dataset, col="Species")
g = g.map(plt.hist, "SepalWidthCm")
g = sns.FacetGrid(iris_dataset, col="Species")
g = g.map(plt.hist, "PetalLengthCm")
g = sns.FacetGrid(iris_dataset, col="Species")
g = g.map(plt.hist, "PetalWidthCm")

Summary Q1 :¶

Looking at the above histograms with class lables we can see that the PetalWidthCm clearly distinguish the three species best.
We can see this based on the differnt bins of the histograms for the three species for PetalWidhtCm, which are cleary seperate which indicates that we can distinguish the species by PetalWidhtCm range of values.
The range for PetalWidhtCm is as below for the three different species :

PetalWidthCm¶

    * Iris-setosa :  0.1 to 0.6
    * Iris-versicolor : 1.0 to 1.8
    * Iris-virginica : 1.4 to 2.5

PetalLengthCm is not the candidate as there is some overlap between the values for Iris-versicolor and Iris-virginica, hence its is not the one

PetalLengthCm

  * Iris-setosa : 0.1 to 1.9
  * Iris-versicolor : 3.0 to 5.1
  * Iris-virginica : 4.5 to 6.9

#Extra distribution
#pandas_profiling.ProfileReport(iris_dataset)
#Extra

Question 2 :¶

#loading csv
#2
boson_dataset = pd.read_csv('/Users/rajmati.marlecha/Desktop/DMGAssignment/higgs-boson/training.csv')
boson_dataset.head()

#loading csv
boson_dataset_test = pd.read_csv('/Users/rajmati.marlecha/Desktop/DMGAssignment/higgs-boson/test.csv')
boson_dataset_test.head()

boson_dataset.columns.size

33

boson_dataset.Label.unique()

array(['s', 'b'], dtype=object)

print(boson_dataset[boson_dataset.columns.difference(['EventId', 'Label'])].columns)

Index(['DER_deltaeta_jet_jet', 'DER_deltar_tau_lep', 'DER_lep_eta_centrality',
       'DER_mass_MMC', 'DER_mass_jet_jet', 'DER_mass_transverse_met_lep',
       'DER_mass_vis', 'DER_met_phi_centrality', 'DER_prodeta_jet_jet',
       'DER_pt_h', 'DER_pt_ratio_lep_tau', 'DER_pt_tot', 'DER_sum_pt',
       'PRI_jet_all_pt', 'PRI_jet_leading_eta', 'PRI_jet_leading_phi',
       'PRI_jet_leading_pt', 'PRI_jet_num', 'PRI_jet_subleading_eta',
       'PRI_jet_subleading_phi', 'PRI_jet_subleading_pt', 'PRI_lep_eta',
       'PRI_lep_phi', 'PRI_lep_pt', 'PRI_met', 'PRI_met_phi', 'PRI_met_sumet',
       'PRI_tau_eta', 'PRI_tau_phi', 'PRI_tau_pt', 'Weight'],
      dtype='object')

#pandas_profiling.ProfileReport(boson_dataset)

#histogram without classes for all the columns of train
hist = boson_dataset[boson_dataset.columns.difference(['EventId', 'Label','Weight'])].hist(figsize=(30,30))

# the entire dataset including the test
#result_df = pd.DataFrame(boson_dataset.append(boson_dataset_test))
#hist = result_df[result_df.columns.difference(['EventId', 'Label',''])].hist(figsize=(30,30))

boson_dataset_s = boson_dataset[boson_dataset.Label=='s']
boson_dataset_b = boson_dataset[boson_dataset.Label=='b']
print('Total: ' + str(boson_dataset.size))
print('Class S: ' + str(boson_dataset_s.size))
print('Class B: '+ str(boson_dataset_b.size))

Total: 8250000
Class S: 2827011
Class B: 5422989

hist_s = boson_dataset_s[boson_dataset_s.columns.difference(['EventId', 'Label','Weight'])].hist(figsize=(30,30))

hist_b = boson_dataset_b[boson_dataset_b.columns.difference(['EventId', 'Label','Weight'])].hist(figsize=(30,30))

for i, col in enumerate(boson_dataset.columns.difference(['EventId', 'Label','Weight'])):
    plt.figure(i)
    sns.distplot(boson_dataset[boson_dataset.Label=='s'][col])
    sns.distplot(boson_dataset[boson_dataset.Label=='b'][col])

Summary Q2 :¶

Looking at the above histograms , none of the variables can be said to have clear distinction between values
Each variable has a decent overlap with respect to the classes
Even the log values wont help in distinguising them
We can see that the ranges for DER_mass_transverse_met_lep are distinguishable however there is a big overlap there too

Question 3¶

iris_dataset.head()

g = sns.FacetGrid(iris_dataset, hue="Species",height=(6))
g = (g.map(plt.scatter, "SepalLengthCm", "SepalWidthCm").add_legend())
g = sns.FacetGrid(iris_dataset,  hue="Species",height=(6))
g = (g.map(plt.scatter, "SepalLengthCm", "PetalWidthCm").add_legend())
g = sns.FacetGrid(iris_dataset,  hue="Species",height=(6))
g = (g.map(plt.scatter, "SepalWidthCm", "PetalLengthCm").add_legend())
g = sns.FacetGrid(iris_dataset,  hue="Species",height=(6))
g = (g.map(plt.scatter, "SepalWidthCm", "PetalWidthCm").add_legend())
g = sns.FacetGrid(iris_dataset,  hue="Species",height=(6))
g = (g.map(plt.scatter, "SepalLengthCm", "PetalLengthCm").add_legend())
g = sns.FacetGrid(iris_dataset,  hue="Species",height=(6))
g = (g.map(plt.scatter, "PetalLengthCm", "PetalWidthCm").add_legend())

Summary Q3 : Looking at the above scatter plots its evident that the classes are well seperated to the best extent in the below combination of scatter plot as it shows maximum seperation amongth the species:¶

Petal WidthCm and PetalLengthCm
There are other two as well from the bottom up , however there is overlap between the points there

Question 4 :¶

##This block considers all variables and can be used iteratively to minimize re-running code
#We can limit the variables of interest if we want to
def Plot(varname,dataset):   
    f, (ax1,ax2) = plt.subplots(1, 2, figsize=(20,4))
    
    ax1.set_title('Before Log for var:' + varname)
    ax2.set_title('After Log for var:' + varname)
    
    sns.distplot(dataset[varname].fillna(0),ax=ax1,fit=norm)   
    sns.distplot((np.log(dataset[varname]).fillna(0)),ax=ax2,fit=norm)
    
    plt.show()
   


Plot('PRI_met',boson_dataset)      
Plot('DER_mass_vis',boson_dataset)    
Plot('DER_pt_ratio_lep_tau',boson_dataset)
Plot('PRI_met_sumet',boson_dataset)


boson_dataset['PRI_met_log'] = np.log(boson_dataset['PRI_met'])
boson_dataset['DER_mass_vis_log']= np.log(boson_dataset['DER_mass_vis'])
boson_dataset['DER_pt_ratio_lep_tau_log']= np.log(boson_dataset['DER_pt_ratio_lep_tau'])
boson_dataset['PRI_met_sumet_log']= np.log(boson_dataset['PRI_met_sumet'])

boson_dataset_sample = boson_dataset.groupby('Label').apply(lambda s: s.sample(2000)) #boson_dataset.sample(n=10)
g = sns.FacetGrid(boson_dataset_sample, hue="Label",height=(6))
g = (g.map(plt.scatter, "PRI_met_sumet", "DER_pt_ratio_lep_tau").add_legend())

g = sns.FacetGrid(boson_dataset_sample, hue="Label",height=(6))
g = (g.map(plt.scatter, "PRI_met_sumet_log", "DER_pt_ratio_lep_tau_log").add_legend())

#Additional
g = sns.FacetGrid(boson_dataset_sample, hue="Label",height=(6))
g = (g.map(plt.scatter, "PRI_met", "DER_mass_vis").add_legend())

g = sns.FacetGrid(boson_dataset_sample, hue="Label",height=(6))
g = (g.map(plt.scatter, "PRI_met_log", "DER_mass_vis_log").add_legend())

Summary Q4 :¶

Picked up those columns which after log transformation could be visualized as a normal distribution post log as shown above in individual histograms of variables without and with log.
Choose combination of two to plot scatter before and after log.
Looking at the above scatter plots before log and after log for the variables "PRI_met_sumet" and "DER_pt_ratio_lep_tau" we see that
- Looking at the plain scatter plots we cant distinguish the classes much , moreover there doesnt seem to be a relation between the two variables, and they are very closely overlapping
- Normalizing the data using log has made the data spread out and visually interpretable
- Looking at the logged scatter plots of the same variables we can see a negative correlation getting established between PRI_met_sumet and DER_pt_ratio_lep_tau

Question 5 : 2D PCA for each digit class¶

#loading csv
digit_recognizer = pd.read_csv('/Users/rajmati.marlecha/Desktop/DMGAssignment/digit-recognizer/train.csv')
digit_recognizer.shape

(42000, 785)

digitArray = digit_recognizer.label.unique()
digitArray.sort()
digitArray

array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])

def twoDPCA(clas):
    # PCA with 2 components.
    pca = PCA(n_components = 2)
    #print(pca)
    #Fitting PCA to the iris dataset and transforming it into 2 principal components
    X, y = digit_recognizer[digit_recognizer.label==clas].iloc[:, 1:].values, digit_recognizer.iloc[:, 0].values
    #Standardize the data first
    from sklearn.preprocessing import StandardScaler
    sc = StandardScaler()
    X_std = sc.fit_transform(X)

    X_proj = pca.fit_transform(X_std)
    return X_proj

def plot2DPCA(digits):
    f, (ax1,ax2,ax3) = plt.subplots(1,3, figsize=(20,4))
    # Plotting the projected principal components and try to understand the data. 
    # c=y colors the scatter plot based on y (target)
    X = twoDPCA(digits[0])
    ax1.set_title('Scatter for 2D PCA for Digit :' + str(digits[0]))
    ax1.scatter(X[:,0], X[:,1])
    if digits.size > 2:
        Y = twoDPCA(digits[1])
        Z = twoDPCA(digits[2])
        ax2.set_title('Scatter for 2D PCA for Digit :' + str(digits[1]))
        ax3.set_title('Scatter for 2D PCA for Digit :' + str(digits[2]))
        ax2.scatter(Y[:,0], Y[:,1])
        ax3.scatter(Z[:,0], Z[:,1])
    #plt.figure(figsize = (10,8))
    #plt.colorbar()
    plt.show()

for x in range(0,digitArray.size,3):
    #print(x)
    y=x+3
    plot2DPCA(digitArray[x:y:])

Question 5 Summary :¶

* Looking at the above scatter plots of 2D PCA for digits we can see 
* 1, 4 ,7 , 9 are the digits that have more spread as compared to the other digits 
* We can observe this by looking at the range of the values of the PCA components
* We can deduce that there is a higher spread amongst the numbers that can be written in multiple ways
* People write these digits in different ways

Question 6¶

pca = PCA(n_components = 30)
X = digit_recognizer.iloc[:, 1:785].values
DIGITS_PCA_30_dataset_array = pca.fit_transform(X)
DIGITS_PCA_30_dataset =pd.DataFrame(DIGITS_PCA_30_dataset_array)
DIGITS_PCA_30_dataset.head()

DIGITS_PCA_30_dataset['label']=digit_recognizer['label']
print(DIGITS_PCA_30_dataset.shape)
DIGITS_PCA_30_dataset.head()

(42000, 31)

cluster_centers_class0 = DIGITS_PCA_30_dataset[DIGITS_PCA_30_dataset.label==0].iloc[:, 0:30].sample(n=10).reset_index(drop=True)
cluster_centers_class1 = DIGITS_PCA_30_dataset[DIGITS_PCA_30_dataset.label==1].iloc[:, 0:30].sample(n=10).reset_index(drop=True)
cluster_centers_class2 = DIGITS_PCA_30_dataset[DIGITS_PCA_30_dataset.label==2].iloc[:, 0:30].sample(n=10).reset_index(drop=True)
cluster_centers_class3 = DIGITS_PCA_30_dataset[DIGITS_PCA_30_dataset.label==3].iloc[:, 0:30].sample(n=10).reset_index(drop=True)
cluster_centers_class4 = DIGITS_PCA_30_dataset[DIGITS_PCA_30_dataset.label==4].iloc[:, 0:30].sample(n=10).reset_index(drop=True)
cluster_centers_class5 = DIGITS_PCA_30_dataset[DIGITS_PCA_30_dataset.label==5].iloc[:, 0:30].sample(n=10).reset_index(drop=True)
cluster_centers_class6 = DIGITS_PCA_30_dataset[DIGITS_PCA_30_dataset.label==6].iloc[:, 0:30].sample(n=10).reset_index(drop=True)
cluster_centers_class7 = DIGITS_PCA_30_dataset[DIGITS_PCA_30_dataset.label==7].iloc[:, 0:30].sample(n=10).reset_index(drop=True)
cluster_centers_class8 = DIGITS_PCA_30_dataset[DIGITS_PCA_30_dataset.label==8].iloc[:, 0:30].sample(n=10).reset_index(drop=True)
cluster_centers_class9 = DIGITS_PCA_30_dataset[DIGITS_PCA_30_dataset.label==9].iloc[:, 0:30].sample(n=10).reset_index(drop=True)
cluster_centers_class0.head(10)

#picking one cluster center from each class
cluster_centers_allclasses = DIGITS_PCA_30_dataset.groupby('label').apply(lambda x: x.sample(1)).reset_index(drop=True)
cluster_centers_allclasses.drop(['label'], axis=1,inplace=True)
cluster_centers_allclasses.head()

def checkPurity(y_rand, y_pred_rand):
    mat_rand= metrics.confusion_matrix(y_pred_rand, y_rand)
    #print(mat_rand)
    #print(metrics.accuracy_score(y, y_pred_rand))
    maxmat_rand = mat_rand.max(axis=1)
    #print(mat_rand.max(axis=1))
    #print(maxmat_rand.sum())
    purity = maxmat_rand.sum()/mat_rand.sum()
    return round(purity,3)

def runKmeanswithInitData(df_init):    
    ktest = KMeans(n_clusters=10,init=df_init,max_iter=200,tol=0.0001)
    
    X,y = DIGITS_PCA_30_dataset.iloc[:, 0:30],DIGITS_PCA_30_dataset.iloc[:, 30]
    # fitting the model to X 
    fitted = ktest.fit(X)
    # predicting labels (y) and saving to y_pred
    y_pred = ktest.predict(X)
    return y,y_pred,fitted.n_iter_


def RunAll():
    clusts_inits = []
    iters_inits = []
    purity_inits = []

    print("Clusters with Inits for all different Initializations")
   
    clusts_inits.append(0)
    y,y_pred,n_iter_ = runKmeanswithInitData(cluster_centers_class0)
    iters_inits.append(n_iter_)
    pureVal_inits = checkPurity(y,y_pred)
    purity_inits.append(pureVal_inits)
    
    clusts_inits.append(1)
    y,y_pred,n_iter_ = runKmeanswithInitData(cluster_centers_class1)
    iters_inits.append(n_iter_)
    pureVal_inits = checkPurity(y,y_pred)
    purity_inits.append(pureVal_inits)
    
    
    clusts_inits.append(2)
    y,y_pred,n_iter_ = runKmeanswithInitData(cluster_centers_class2)
    iters_inits.append(n_iter_)
    pureVal_inits = checkPurity(y,y_pred)
    purity_inits.append(pureVal_inits)
    
    
    clusts_inits.append(3)
    y,y_pred,n_iter_ = runKmeanswithInitData(cluster_centers_class3)
    iters_inits.append(n_iter_)
    pureVal_inits = checkPurity(y,y_pred)
    purity_inits.append(pureVal_inits)
    
    
    clusts_inits.append(4)
    y,y_pred,n_iter_ = runKmeanswithInitData(cluster_centers_class4)
    iters_inits.append(n_iter_)
    pureVal_inits = checkPurity(y,y_pred)
    purity_inits.append(pureVal_inits)
    
    
    clusts_inits.append(5)
    y,y_pred,n_iter_ = runKmeanswithInitData(cluster_centers_class5)
    iters_inits.append(n_iter_)
    pureVal_inits = checkPurity(y,y_pred)
    purity_inits.append(pureVal_inits)
    
    
    clusts_inits.append(6)
    y,y_pred,n_iter_ = runKmeanswithInitData(cluster_centers_class6)
    iters_inits.append(n_iter_)
    pureVal_inits = checkPurity(y,y_pred)
    purity_inits.append(pureVal_inits)
    
    clusts_inits.append(7)
    y,y_pred,n_iter_ = runKmeanswithInitData(cluster_centers_class7)
    iters_inits.append(n_iter_)
    pureVal_inits = checkPurity(y,y_pred)
    purity_inits.append(pureVal_inits)
    
    
    clusts_inits.append(8)
    y,y_pred,n_iter_ = runKmeanswithInitData(cluster_centers_class8)
    iters_inits.append(n_iter_)
    pureVal_inits = checkPurity(y,y_pred)
    purity_inits.append(pureVal_inits)
    
    clusts_inits.append(9)
    y,y_pred,n_iter_ = runKmeanswithInitData(cluster_centers_class9)
    iters_inits.append(n_iter_)
    pureVal_inits = checkPurity(y,y_pred)
    purity_inits.append(pureVal_inits)
    
    clusts_inits.append('All')
    y,y_pred,n_iter_ = runKmeanswithInitData(cluster_centers_allclasses)
    iters_inits.append(n_iter_)
    pureVal_inits = checkPurity(y,y_pred)
    purity_inits.append(pureVal_inits)
    
    data = pd.DataFrame()
    data['initialization_method_class'] = clusts_inits
    data['iterations'] = iters_inits
    data['purity_inits'] = purity_inits
    return data
df = RunAll()  
df

Clusters with Inits for all different Initializations

Question 6A Summary :¶

Looking at the above table output we can see that we have the best purity score when the clusters are initialized from different classes : (0.632) a.k.a each individual class center as the 10 cluster centers
the All class run also took 73 iterations to converge at the given tolerance stopping criteria.
We also have a decent purity 0.598 when the centers are initialized from class 3 , with convergence at just 32 iterations

Question 6B :¶

def runKmeansRandomInit(n):
    # number of clusters to predict 3
    
    ktest_rand = KMeans(n_clusters=n,random_state=0)
    X_rand,y_rand = DIGITS_PCA_30_dataset.iloc[:, 0:30],DIGITS_PCA_30_dataset.iloc[:, 30]
    # fitting the model to X 
    fitted_rand = ktest_rand.fit(X_rand)
    # predicting labels (y) and saving to y_pred
    y_pred_rand = ktest_rand.predict(X_rand)
    return y_rand,y_pred_rand,fitted_rand.n_iter_

clusts = []
purity = []
for noofClusters in range(5,30,5):
    print("Cluster")
    print(noofClusters)
    clusts.append(noofClusters)
    y_rand,y_pred_rand,iters = runKmeansRandomInit(noofClusters)
    #print(iters)
    pureVal = checkPurity(y_rand,y_pred_rand)
    purity.append(pureVal)

Cluster
5
Cluster
10
Cluster
15
Cluster
20
Cluster
25

data = pd.DataFrame()
data['clusters'] = clusts
data['purity'] = purity
ax = sns.scatterplot(x="clusters", y="purity", data=data)

Question 6B Interpretation¶

Looking at above plot for different cluster sizes we can observe that as we increase the number of clusters the purity increases

Question 7 :¶

#We will use the data without the labels as input for GMM
iris_dataset.iloc[:, 1:5].head()

#Created function to fit the gaussian lAdded comments in the function  

from sklearn.mixture import GaussianMixture

def runGMM(mixture):
    gmm = GaussianMixture(n_components=mixture,random_state=0,covariance_type='full')
    fit = gmm.fit(iris_dataset.iloc[:, 1:5])
    #both the score with the training data passed and the lower_bound_ would return the same log likelihood 
    #values that we can the take exponent of which gives us the likelihood values that we later use for plotting
    #print("score")
    #gmm.score(iris_dataset.iloc[:, 1:5])
    #print("lower bound")
    return gmm.lower_bound_


mixtures = []
likelihood = []

for mix in range(1,33,2):
    #print("Mixtures")
    #print(mix)
    mixtures.append(mix)
    likeliVal =round(np.exp(runGMM(mix)),2)
    likelihood.append(likeliVal)
    #print("Likeli Val")
    #print(likeliVal)
    
data = pd.DataFrame()
data['mixtures'] = mixtures
data['likelihood'] = likelihood
ax = sns.scatterplot(x="mixtures", y="likelihood", data=data)    
ax.set(xticks=np.arange(1,35,2))

[[<matplotlib.axis.XTick at 0x1a2ae9e550>,
  <matplotlib.axis.XTick at 0x1a1c8b7a58>,
  <matplotlib.axis.XTick at 0x1a1d591668>,
  <matplotlib.axis.XTick at 0x1a29f93dd8>,
  <matplotlib.axis.XTick at 0x1a29f93fd0>,
  <matplotlib.axis.XTick at 0x1a29f93b38>,
  <matplotlib.axis.XTick at 0x1a2ae85710>,
  <matplotlib.axis.XTick at 0x1a2ae85f28>,
  <matplotlib.axis.XTick at 0x1a2ae85470>,
  <matplotlib.axis.XTick at 0x1a1f5afb70>,
  <matplotlib.axis.XTick at 0x1a1d886da0>,
  <matplotlib.axis.XTick at 0x1a2ae85cc0>,
  <matplotlib.axis.XTick at 0x1a1d88d208>,
  <matplotlib.axis.XTick at 0x1a218d9438>,
  <matplotlib.axis.XTick at 0x1a218d9080>,
  <matplotlib.axis.XTick at 0x1a218d9cf8>,
  <matplotlib.axis.XTick at 0x1a1cd10080>]]

Question 7 Summary:¶

Looking at the above plot we can see the likelihood value against no of mixtures used of gaussian.
The likelihood value will increase as we increase the number of mixture components
This is expected as we try to increase the likelihood by better fitting the data

Number of variables	6
Number of observations	150
Total Missing (%)	0.0%
Total size in memory	7.1 KiB
Average record size in memory	48.5 B

Numeric	4
Categorical	1
Boolean	0
Date	0
Text (Unique)	0
Rejected	1
Unsupported	0

Distinct count	150
Unique (%)	100.0%
Missing (%)	0.0%
Missing (n)	0
Infinite (%)	0.0%
Infinite (n)	0

Mean	75.5
Minimum	1
Maximum	150
Zeros (%)	0.0%

Minimum	1
5-th percentile	8.45
Q1	38.25
Median	75.5
Q3	112.75
95-th percentile	142.55
Maximum	150
Range	149
Interquartile range	74.5

Question 1 :¶

Summary Q1 :¶

PetalWidthCm¶

Overview

Variables

Correlations

Sample

Question 2 :¶

Overview

Variables

Correlations

Sample

Summary Q2 :¶

Question 3¶

Summary Q3 : Looking at the above scatter plots its evident that the classes are well seperated to the best extent in the below combination of scatter plot as it shows maximum seperation amongth the species:¶

Question 4 :¶

Summary Q4 :¶

Question 5 : 2D PCA for each digit class¶

Question 5 Summary :¶

Question 6¶

Question 6A Summary :¶

Question 6B :¶

Question 6B Interpretation¶

Question 7 :¶

Question 7 Summary:¶

	Id	SepalLengthCm	SepalWidthCm	PetalLengthCm	PetalWidthCm	Species
0	1	5.1	3.5	1.4	0.2	Iris-setosa
1	2	4.9	3.0	1.4	0.2	Iris-setosa
2	3	4.7	3.2	1.3	0.2	Iris-setosa
3	4	4.6	3.1	1.5	0.2	Iris-setosa
4	5	5.0	3.6	1.4	0.2	Iris-setosa

Standard deviation	43.445
Coef of variation	0.57544
Kurtosis	-1.2
Mean	75.5
MAD	37.5
Skewness	0
Sum	11325
Variance	1887.5
Memory size	1.2 KiB

Value	Count	Frequency (%)
150	1	0.7%
56	1	0.7%
54	1	0.7%
53	1	0.7%
52	1	0.7%
51	1	0.7%
50	1	0.7%
49	1	0.7%
48	1	0.7%
47	1	0.7%
Other values (140)	140	93.3%

Distinct count	43
Unique (%)	28.7%
Missing (%)	0.0%
Missing (n)	0
Infinite (%)	0.0%
Infinite (n)	0

Standard deviation	1.7644
Coef of variation	0.46943
Kurtosis	-1.4019
Mean	3.7587
MAD	1.5619
Skewness	-0.27446
Sum	563.8
Variance	3.1132
Memory size	1.2 KiB

Distinct count	35
Unique (%)	23.3%
Missing (%)	0.0%
Missing (n)	0
Infinite (%)	0.0%
Infinite (n)	0

Standard deviation	0.82807
Coef of variation	0.14171
Kurtosis	-0.55206
Mean	5.8433
MAD	0.68756
Skewness	0.31491
Sum	876.5
Variance	0.68569
Memory size	1.2 KiB

Distinct count	23
Unique (%)	15.3%
Missing (%)	0.0%
Missing (n)	0
Infinite (%)	0.0%
Infinite (n)	0

Minimum	2
5-th percentile	2.345
Q1	2.8
Median	3
Q3	3.3
95-th percentile	3.8
Maximum	4.4
Range	2.4
Interquartile range	0.5

Standard deviation	0.43359
Coef of variation	0.14198
Kurtosis	0.29078
Mean	3.054
MAD	0.33309
Skewness	0.33405
Sum	458.1
Variance	0.188
Memory size	1.2 KiB

Value	Count	Frequency (%)
1.5	14	9.3%
1.4	12	8.0%
5.1	8	5.3%
4.5	8	5.3%
1.3	7	4.7%
1.6	7	4.7%
5.6	6	4.0%
4.0	5	3.3%
4.9	5	3.3%
4.7	5	3.3%
Other values (33)	73	48.7%

Value	Count	Frequency (%)
5.0	10	6.7%
6.3	9	6.0%
5.1	9	6.0%
6.7	8	5.3%
5.7	8	5.3%
5.5	7	4.7%
5.8	7	4.7%
6.4	7	4.7%
6.0	6	4.0%
4.9	6	4.0%
Other values (25)	73	48.7%

Value	Count	Frequency (%)
3.0	26	17.3%
2.8	14	9.3%
3.2	13	8.7%
3.4	12	8.0%
3.1	12	8.0%
2.9	10	6.7%
2.7	9	6.0%
2.5	8	5.3%
3.5	6	4.0%
3.8	6	4.0%
Other values (13)	34	22.7%

Value	Count	Frequency (%)
2.0	1	0.7%
2.2	3	2.0%
2.3	4	2.7%
2.4	3	2.0%
2.5	8	5.3%

	EventId	DER_mass_MMC	DER_mass_transverse_met_lep	DER_mass_vis	DER_pt_h	DER_deltaeta_jet_jet	DER_mass_jet_jet	DER_prodeta_jet_jet	DER_deltar_tau_lep	DER_pt_tot	...	PRI_jet_num	PRI_jet_leading_pt	PRI_jet_leading_eta	PRI_jet_leading_phi	PRI_jet_subleading_pt	PRI_jet_subleading_eta	PRI_jet_subleading_phi	PRI_jet_all_pt	Weight	Label
0	100000	138.470	51.655	97.827	27.980	0.91	124.711	2.666	3.064	41.928	...	2	67.435	2.150	0.444	46.062	1.24	-2.475	113.497	0.002653	s
1	100001	160.937	68.768	103.235	48.146	-999.00	-999.000	-999.000	3.473	2.078	...	1	46.226	0.725	1.158	-999.000	-999.00	-999.000	46.226	2.233584	b
2	100002	-999.000	162.172	125.953	35.635	-999.00	-999.000	-999.000	3.148	9.336	...	1	44.251	2.053	-2.028	-999.000	-999.00	-999.000	44.251	2.347389	b
3	100003	143.905	81.417	80.943	0.414	-999.00	-999.000	-999.000	3.310	0.414	...	0	-999.000	-999.000	-999.000	-999.000	-999.00	-999.000	-0.000	5.446378	b
4	100004	175.864	16.915	134.805	16.405	-999.00	-999.000	-999.000	3.891	16.405	...	0	-999.000	-999.000	-999.000	-999.000	-999.00	-999.000	0.000	6.245333	b

	EventId	DER_mass_MMC	DER_mass_transverse_met_lep	DER_mass_vis	DER_pt_h	DER_deltaeta_jet_jet	DER_mass_jet_jet	DER_prodeta_jet_jet	DER_deltar_tau_lep	DER_pt_tot	...	PRI_met_phi	PRI_met_sumet	PRI_jet_num	PRI_jet_leading_pt	PRI_jet_leading_eta	PRI_jet_leading_phi	PRI_jet_subleading_pt	PRI_jet_subleading_eta	PRI_jet_subleading_phi	PRI_jet_all_pt
0	350000	-999.000	79.589	23.916	3.036	-999.000	-999.000	-999.000	0.903	3.036	...	2.022	98.556	0	-999.000	-999.000	-999.000	-999.000	-999.000	-999.000	-0.000
1	350001	106.398	67.490	87.949	49.994	-999.000	-999.000	-999.000	2.048	2.679	...	-1.138	176.251	1	47.575	-0.553	-0.849	-999.000	-999.000	-999.000	47.575
2	350002	117.794	56.226	96.358	4.137	-999.000	-999.000	-999.000	2.755	4.137	...	-1.868	111.505	0	-999.000	-999.000	-999.000	-999.000	-999.000	-999.000	0.000
3	350003	135.861	30.604	97.288	9.104	-999.000	-999.000	-999.000	2.811	9.104	...	1.172	164.707	0	-999.000	-999.000	-999.000	-999.000	-999.000	-999.000	0.000
4	350004	74.159	82.772	58.731	89.646	1.347	536.663	-0.339	1.028	77.213	...	-0.231	869.614	3	254.085	-1.013	-0.334	185.857	0.335	2.587	599.213

Number of variables	37
Number of observations	250000
Total Missing (%)	0.0%
Total size in memory	70.6 MiB
Average record size in memory	296.0 B

Distinct count	7087
Unique (%)	2.8%
Missing (%)	0.0%
Missing (n)	0
Infinite (%)	0.0%
Infinite (n)	0

Minimum	-999
5-th percentile	-999
Q1	-999
Median	-999
Q3	0.49
95-th percentile	4.276
Maximum	8.503
Range	1007.5
Interquartile range	999.49

Standard deviation	454.48
Coef of variation	-0.64154
Kurtosis	-1.1449
Mean	-708.42
MAD	412.52
Skewness	0.92469
Sum	-177110000
Variance	206550
Memory size	1.9 MiB

Distinct count	4692
Unique (%)	1.9%
Missing (%)	0.0%
Missing (n)	0
Infinite (%)	0.0%
Infinite (n)	0

Minimum	0.208
5-th percentile	0.973
Q1	1.81
Median	2.4915
Q3	2.961
95-th percentile	3.441
Maximum	5.684
Range	5.476
Interquartile range	1.151

Standard deviation	0.78291
Coef of variation	0.32991
Kurtosis	-0.22245
Mean	2.3731
MAD	0.64234
Skewness	-0.21578
Sum	593270
Variance	0.61295
Memory size	1.9 MiB

Distinct count	108338
Unique (%)	43.3%
Missing (%)	0.0%
Missing (n)	0
Infinite (%)	0.0%
Infinite (n)	0

Standard deviation	406.35
Coef of variation	-8.2889
Kurtosis	1.6242
Mean	-49.023
MAD	289.66
Skewness	-1.8547
Sum	-12256000
Variance	165120
Memory size	1.9 MiB

Distinct count	101637
Unique (%)	40.7%
Missing (%)	0.0%
Missing (n)	0
Infinite (%)	0.0%
Infinite (n)	0

Minimum	0
5-th percentile	3.389
Q1	19.241
Median	46.524
Q3	73.598
95-th percentile	104.64
Maximum	690.08
Range	690.08
Interquartile range	54.357

Standard deviation	35.345
Coef of variation	0.71781
Kurtosis	6.3668
Mean	49.24
MAD	28.607
Skewness	1.2192
Sum	12310000
Variance	1249.3
Memory size	1.9 MiB

Distinct count	100558
Unique (%)	40.2%
Missing (%)	0.0%
Missing (n)	0
Infinite (%)	0.0%
Infinite (n)	0

Minimum	6.329
5-th percentile	37.874
Q1	59.389
Median	73.752
Q3	92.259
95-th percentile	149.27
Maximum	1349.4
Range	1343
Interquartile range	32.87

Standard deviation	40.829
Coef of variation	0.50293
Kurtosis	35.494
Mean	81.182
MAD	25.454
Skewness	3.7903
Sum	20295000
Variance	1667
Memory size	1.9 MiB

Distinct count	2829
Unique (%)	1.1%
Missing (%)	0.0%
Missing (n)	0
Infinite (%)	0.0%
Infinite (n)	0

Minimum	-1.414
5-th percentile	-1.413
Q1	-1.371
Median	-0.356
Q3	1.225
95-th percentile	1.412
Maximum	1.414
Range	2.828
Interquartile range	2.596

Standard deviation	1.1936
Coef of variation	-9.3027
Kurtosis	-1.7681
Mean	-0.1283
MAD	1.1308
Skewness	0.15114
Sum	-32076
Variance	1.4246
Memory size	1.9 MiB